#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""
Data pre-processing: build vocabularies and binarize training data.
"""

import argparse
import codecs
import glob
import os
import random

import soundfile
import tqdm


def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "root", metavar="DIR", help="root directory containing flac files to index"
    )
    parser.add_argument(
        "--valid-percent",
        default=0.01,
        type=float,
        metavar="D",
        help="percentage of data to use as validation set (between 0 and 1)",
    )
    parser.add_argument(
        "--dest", default=".", type=str, metavar="DIR", help="output directory"
    )
    parser.add_argument(
        "--ext", default="flac", type=str, metavar="EXT", help="extension to look for"
    )
    parser.add_argument("--seed", default=42, type=int, metavar="N", help="random seed")
    parser.add_argument(
        "--path-must-contain",
        default=None,
        type=str,
        metavar="FRAG",
        help="if set, path must contain this substring for a file to be included in the manifest",
    )
    parser.add_argument(
        "--part-tsv-path",
        default=None,
        type=str,
        metavar="FRAG",
        help="设置了这个后，改tsv文件中包含的内容不再提取",
    )
    parser.add_argument(
        "--part-tsv-path2",
        default=None,
        type=str,
        metavar="FRAG",
        help="设置了这个后，改tsv文件中包含的内容不再提取",
    )
    parser.add_argument(
        "--split",
        default="train",
        type=str,
        metavar="FRAG",
        help="得到tsv的命名",
    )
    return parser

def load_list_file_clean(path: str):
    """
    得到不包含换行符的str_list
    :param path:
    :return:
    """
    if not os.path.exists(path):
        print(f'load_list_file_clean()_{path}文件不存在')
        return []

    with codecs.open(path, 'r', encoding='utf-8') as f:
        cat_to_name: list = f.read().splitlines()
        # cat_to_name: list = f.readlines() -> 包含换行符
        print(f"load_list_file_clean()_数据总条数为:{len(cat_to_name)}")
    return cat_to_name
def load_dict_from_tsv(input_tsv_path,is_relpath:bool=True):
    """"""
    print("load_dict_from_tsv", input_tsv_path)
    total_list = load_list_file_clean(input_tsv_path)
    root_dir = total_list[0]
    print("root_dir:", root_dir)
    left_list = total_list[1:]
    res_dict = {}
    for line in left_list:
        line_list = line.strip().split()
        assert len(line_list) == 2  # tsv的格式硬性要求
        if is_relpath:
            key = line_list[0]
        else:
            key = os.path.join(root_dir, line_list[0])
        res_dict[key] = line_list[1]
    return res_dict

def main(args):
    assert args.valid_percent >= 0 and args.valid_percent <= 1.0

    if not os.path.exists(args.dest):
        os.makedirs(args.dest)

    dir_path = os.path.realpath(args.root)
    search_path = os.path.join(dir_path, "**/*." + args.ext)
    rand = random.Random(args.seed)

    valid_f = (
        open(os.path.join(args.dest, "valid.tsv"), "w")
        if args.valid_percent > 0
        else None
    )
    if args.part_tsv_path is not None:
        part_dict = load_dict_from_tsv(args.part_tsv_path, False)
    else:
        part_dict = {}
    if args.part_tsv_path2 is not None:
        part_dict2 = load_dict_from_tsv(args.part_tsv_path2, False)
        part_dict.update(part_dict2)

    with open(os.path.join(args.dest, "train.tsv"), "w") as train_f:
        print(dir_path, file=train_f)

        if valid_f is not None:
            print(dir_path, file=valid_f)
        file_list = glob.glob(search_path, recursive=True)
        tmp_list = [] # 每50万条一写
        for fname in tqdm.tqdm(file_list, total=len(file_list)):
            file_path = os.path.realpath(fname)
            if args.part_tsv_path is not None:
                if file_path in part_dict:
                    print("skip:", file_path)
                    continue

            if args.path_must_contain and args.path_must_contain not in file_path:
                continue

            frames = soundfile.info(fname).frames
            res = "{}\t{}".format(os.path.relpath(file_path, dir_path), frames)
            tmp_list.append(res)
            if len(tmp_list) == 500000:
                for res_line in tmp_list:
                    if valid_f is None:
                        dest = train_f
                    else:
                        dest = train_f if rand.random() > args.valid_percent else valid_f
                    print(
                        res_line, file=dest
                    )
                tmp_list = []
        if len(tmp_list) != 0:
            for res_line in tmp_list:
                if valid_f is None:
                    dest = train_f
                else:
                    dest = train_f if rand.random() > args.valid_percent else valid_f
                print(
                    res_line, file=dest
                )
    if valid_f is not None:
        valid_f.close()


if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()
    main(args)
